/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
 *
 * File Name     : roce_mix.c
 * Version       : v2.0
 * Created       : 2021/3/12
 * Last Modified : 2021/12/7
 * Description   : The definition of callback functions and internal functions of RoCE mix features.
 */

#include <rdma/ib_mad.h>
#include <rdma/ib_verbs.h>
#if defined(CENTOS8_2_4_18_0)
#include <rdma/uverbs_ioctl.h>
#endif
#include <linux/pci.h>

#include "securec.h" /* Huawei secure functions; */

#include <rdma/ib_addr.h>
#include <rdma/ib_cache.h>

#include "hinic3_crm.h"
#include "hinic3_srv_nic.h"

#include "roce.h"
#include "roce_compat.h"
#include "roce_user.h"
#include "roce_pd.h"
#include "roce_qp.h"
#include "roce_cmd.h"
#include "roce_gid.h"
#include "roce_netdev.h"
#include "roce_main_extension.h"
#include "roce_npu_cmd.h"
#include "roce_mix.h"

#ifdef ROCE_BONDING_EN
#include "roce_bond.h"
#endif

#ifdef LLT_TEST
#include "llt_roce_stub.h"
#endif
#if defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0))
struct net_device *roce3_ib_get_netdev(struct ib_device *ibdev, u32 port_num)
#else
struct net_device *roce3_ib_get_netdev(struct ib_device *ibdev, u8 port_num)
#endif
{
    struct roce3_device *rdev = NULL;
    struct net_device *netdev = NULL;

    if (ibdev == NULL) {
        pr_err("[ROCE] %s: Ibdev is null \n", __func__);
        return NULL;
    }

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        pr_err("[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__, rdev->glb_func_id);
        return NULL;
    }

#ifdef ROCE_BONDING_EN
    netdev = roce3_bond_get_netdev(rdev);
    if (netdev != NULL) {
        return netdev;
    }
#endif

    netdev = rdev->ndev;

    dev_hold(netdev);

    return netdev;
}

static void roce3_parse_fw_version(struct roce3_device *rdev, u64 *fw_ver)
{
    int ret;
    int i = 0;
    struct hinic3_fw_version fw_version;
    char *fw_str = (char *)fw_version.microcode_ver;
    char *fw_temp = NULL;
    u64 fw_verion[ROCE_FW_VERSION_LEN] = {0};

    ret = hinic3_get_fw_version(rdev->hwdev, &fw_version, HINIC3_CHANNEL_ROCE);
    if (ret != 0) {
        pr_warn("[ROCE] %s: get fw version failed\n", __func__);
        *fw_ver = ROCE_FW_VER;
        return;
    }
    pr_info("[ROCE] %s: fw ver:%s - %s - %s\n", __func__, fw_version.boot_ver, fw_version.mgmt_ver,
        fw_version.microcode_ver);

    while (((fw_temp = strsep(&fw_str, ".")) != NULL) && (i < ROCE_FW_VERSION_LEN)) {
        ret = sscanf_s(fw_temp, "%llu", &fw_verion[i]);
        if (ret != 1) {
            pr_warn("[ROCE] %s: parse fw version failed\n", __func__);
            *fw_ver = ROCE_FW_VER;
            return;
        }

        i++;
    }

    *fw_ver = (((fw_verion[0] & 0xffffffff) << 32) | // 0 is fw_version array idx, 32 is offset
        ((fw_verion[1] & 0xffff) << 16) |            // 1 is fw_version array idx, 16 is offset
        ((fw_verion[2] & 0xff) << 8) |               // 2 is fw_version array idx, 8 is offset
        (fw_verion[3] & 0xff));                      // 3 is fw_version array idx
}

static void roce3_set_local_cap_flag(const struct rdma_service_cap *rdma_cap, struct ib_device_attr *props)
{
    if (((rdma_cap->flags & RDMA_BMME_FLAG_LOCAL_INV) != 0) && ((rdma_cap->flags & RDMA_BMME_FLAG_REMOTE_INV) != 0) &&
        ((rdma_cap->flags & RDMA_BMME_FLAG_FAST_REG_WR) != 0)) {
        props->device_cap_flags = props->device_cap_flags | IB_DEVICE_MEM_MGT_EXTENSIONS;
    }
}

static void roce3_set_bmme_cap_flag(const struct rdma_service_cap *rdma_cap, struct ib_device_attr *props)
{
    if ((rdma_cap->flags & RDMA_BMME_FLAG_TYPE_2_WIN) != 0) {
        if ((rdma_cap->flags & RDMA_BMME_FLAG_WIN_TYPE_2B) != 0) {
            props->device_cap_flags = props->device_cap_flags | IB_DEVICE_MEM_WINDOW_TYPE_2B;
        } else {
            props->device_cap_flags = props->device_cap_flags | IB_DEVICE_MEM_WINDOW_TYPE_2A;
        }
    }
}

static void roce3_query_device_props_set(struct roce3_device *rdev, struct rdma_service_cap *rdma_cap,
    struct ib_device_attr *props)
{
    props->vendor_id = rdev->pdev->vendor;
    props->vendor_part_id = rdev->pdev->device;
    roce3_parse_fw_version(rdev, &props->fw_ver);
    props->hw_ver = ROCE_HW_VER;

    /* sys_image_guid equal GID */
    props->sys_image_guid = rdev->ib_dev.node_guid;

    props->max_mr_size = ~0ULL;
    props->page_size_cap = rdma_cap->page_size_cap;
    props->max_qp = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_qps - rdma_cap->reserved_qps);
    props->max_qp_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_wqes;
    /*
     * 4.19 ofed will return the smaller of sq/rq sge num to user space.
     * 4.17 We use max_sge to only represent max sq sge num, max_rq_sge is a fixed macro of 16.
     */
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)) || (LINUX_VERSION_CODE == KERNEL_VERSION(4, 19, 90)) \
    || (LINUX_VERSION_CODE == KERNEL_VERSION(4, 18, 0)) || defined(OFED_MLNX_5_7_8)
    props->max_send_sge = rdma_cap->max_sq_sg;
    props->max_recv_sge = rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg;
#else
    props->max_sge = (int)rdma_cap->max_sq_sg;
#endif
    props->max_cq = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs - rdma_cap->reserved_cqs);
    props->max_cqe = (int)rdma_cap->max_cqes;

    if ((rdev->board_info.port_num == ROCE_PORT_NUM_2) && (rdev->board_info.port_speed == ROCE_25G_PORT_SPEED)) {
        // 2 smf for 64B cache
        props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts - rdma_cap->reserved_mrws) / MEND_CAP_DEVIDE;
        props->max_srq =
            (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs - rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs) /
            MEND_CAP_DEVIDE;
    } else {
        props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts - rdma_cap->reserved_mrws);
        props->max_srq =
            (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs - rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs);
    }

    props->max_mw = props->max_mr;
    props->max_pd = (int)(rdma_cap->num_pds - rdma_cap->reserved_pds);
    props->max_qp_rd_atom = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma;
    props->max_qp_init_rd_atom = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma;
    props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;

    props->max_srq_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_wqes;
    props->max_srq_sge = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_sge;
    props->max_fast_reg_page_list_len = rdma_cap->max_frpl_len;
    props->local_ca_ack_delay = (u8)rdma_cap->local_ca_ack_delay;
    props->atomic_cap = ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_ATOMIC) != 0) ? IB_ATOMIC_HCA : IB_ATOMIC_NONE;
    props->masked_atomic_cap = props->atomic_cap;
    props->max_pkeys = (u16)rdma_cap->max_pkeys;
#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) && !defined(OFED_MLNX_5_7_8) && !defined(OFED_MLNX_5_4)
#ifdef ROCE_COMPUTE
    props->max_map_per_fmr = 0;
#else
    props->max_map_per_fmr = (int)rdma_cap->max_fmr_maps;
#endif
    props->max_fmr = props->max_map_per_fmr;
#endif
    props->max_ah = INT_MAX;
    return;
}
/* ****************************************************************************
 Prototype    : roce3_query_device
 Description  : query device attribute
 Input        : struct ib_device *ibdev
                struct ib_device_attr *props
                struct ib_udata *uhw
 Output       : struct ib_device_attr *props

  1.Date         : 2015/5/8
    Modification : Created function

**************************************************************************** */
int roce3_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw)
{
    struct roce3_device *rdev = NULL;
    struct rdma_service_cap *rdma_cap = NULL;
    int ret;

    if ((ibdev == NULL) || (props == NULL)) {
        pr_err("[ROCE] %s: Ibdev or props is null\n", __func__);
        return -EINVAL;
    }

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }

    ret = memset_s(props, sizeof(*props), 0, sizeof(*props));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memset props.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    rdma_cap = &rdev->rdma_cap;

    props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_RC_RNR_NAK_GEN;

    /* APM */
    if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_APM) != 0) {
        props->device_cap_flags = props->device_cap_flags | IB_DEVICE_AUTO_PATH_MIG;
    }

    /* rsvd_lKey */
    if ((rdma_cap->flags & RDMA_BMME_FLAG_RESERVED_LKEY) != 0) {
        props->device_cap_flags = props->device_cap_flags | IB_DEVICE_LOCAL_DMA_LKEY;
    }

    roce3_set_local_cap_flag(rdma_cap, props);

#ifndef ROCE_COMPUTE
    /* support XRC */
    if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_XRC) != 0) {
        props->device_cap_flags = props->device_cap_flags | IB_DEVICE_XRC;
    }
#endif

    /* support MW */
    if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_MEM_WINDOW) != 0) {
        props->device_cap_flags = props->device_cap_flags | IB_DEVICE_MEM_WINDOW;
    }

    roce3_set_bmme_cap_flag(rdma_cap, props);

    roce3_query_device_props_set(rdev, rdma_cap, props);
    return 0;
}

static void eth_link_get_speed(struct ib_port_attr *props, enum mag_cmd_port_speed speed)
{
    switch (speed) {
        /* 10G <==> 1X x 10G */
        case PORT_SPEED_10GB:
            props->active_width = IB_WIDTH_1X;
            props->active_speed = IB_SPEED_QDR;
            break;

        /* 25G <==> 1X x 25G */
        case PORT_SPEED_25GB:
            props->active_width = IB_WIDTH_1X;
            props->active_speed = IB_SPEED_EDR;
            break;

        /* 40G <==> 4X x 10G */
        case PORT_SPEED_40GB:
            props->active_width = IB_WIDTH_4X;
            props->active_speed = IB_SPEED_QDR;
            break;

        /* 100G <==> 4X x 25G */
        case PORT_SPEED_100GB:
            props->active_width = IB_WIDTH_4X;
            props->active_speed = IB_SPEED_EDR;
            break;

        default:
            props->active_width = 0;
            props->active_speed = 0;
            break;
    }
}

static void roce3_set_ib_port_attr(struct ib_port_attr *props, struct roce3_device *rdev)
{
    props->port_cap_flags = IB_PORT_CM_SUP;
    props->gid_tbl_len = (int)rdev->rdma_cap.max_gid_per_port;
    props->max_msg_sz = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz;
    props->pkey_tbl_len = (u16)rdev->rdma_cap.max_pkeys;
    props->max_mtu = IB_MTU_4096;
    props->state = IB_PORT_DOWN;
    props->phys_state = ROCE_PORT_PHYS_STATE_DISABLED;
    props->active_mtu = IB_MTU_256;
}
#if defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4)
static void eth_link_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props)
#else
static void eth_link_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props)
#endif
{
    struct roce3_device *rdev = NULL;
    struct net_device *netdev = NULL;
#ifdef ROCE_BONDING_EN
    struct net_device *upper = NULL;
#endif
    enum ib_mtu mtu;
    enum mag_cmd_port_speed speed = PORT_SPEED_10GB;
    int ret = 0;

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu), dev_name(%s).\n", __func__,
            rdev->glb_func_id, ibdev->name);
        return;
    }

    roce3_set_ib_port_attr(props, rdev);

    ret = hinic3_get_speed(rdev->hwdev, &speed, HINIC3_CHANNEL_ROCE);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get speed, func_id(%d)\n", __func__, rdev->glb_func_id);
        props->active_width = 0;
        props->active_speed = 0;
        return;
    }

    eth_link_get_speed(props, speed);

    netdev = roce3_ib_get_netdev(ibdev, ROCE_DEFAULT_PORT_NUM);
    if (netdev == NULL) {
        return;
    }

#ifdef ROCE_BONDING_EN
    if (roce3_bond_is_active(rdev->pdev)) {
        rcu_read_lock();
        upper = netdev_master_upper_dev_get_rcu(netdev);
        if (upper != NULL) {
            dev_put(netdev);
            netdev = upper;
            dev_hold(netdev);
        }
        rcu_read_unlock();
    }
#endif

    if (netif_running(netdev) && netif_carrier_ok(netdev)) {
        props->state = IB_PORT_ACTIVE;
        props->phys_state = ROCE_PORT_PHYS_STATE_LINKUP;
    }

    mtu = (enum ib_mtu)iboe_get_mtu((int)netdev->mtu);

    dev_put(netdev);

    props->active_mtu = ROCE_MIN(props->max_mtu, mtu);
}

/* ****************************************************************************
 Prototype    : roce3_query_port
 Description  : query port attribute
 Input        : struct ib_device *ibdev
                u8 port
                struct ib_port_attr *props
 Output       : struct ib_port_attr *props

  1.Date         : 2015/5/8
    Modification : Created function

**************************************************************************** */
#if defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
int roce3_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props)
#else
int roce3_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props)
#endif
{
    int ret;
    if ((ibdev == NULL) || (props == NULL)) {
        pr_err("[ROCE] %s: Ibdev or props is null\n", __func__);
        return -EINVAL;
    }

    ret = memset_s(props, sizeof(*props), 0, sizeof(*props));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memset props.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    eth_link_query_port(ibdev, port, props);

    return 0;
}

/* ****************************************************************************
 Prototype    : roce3_query_gid
 Description  : query gid
 Input        : struct ib_device *ibdev
                u8 port
                int index
                union ib_gid *gid
 Output       : union ib_gid *gid

  1.Date         : 2015/5/8
    Modification : Created function

  2.Date         : 2015/6/8
    Modification : Modify function

**************************************************************************** */
#if defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
int roce3_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid)
#else
int roce3_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid)
#endif
{
    int ret = 0;
    struct roce3_device *rdev = NULL;
    struct rdma_gid_entry gid_entry;

    if ((ibdev == NULL) || (gid == NULL)) {
        pr_err("[ROCE] %s: Ibdev or gid is null\n", __func__);
        return -EINVAL;
    }

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }

    ret = roce3_get_gid(rdev->hwdev, (u32)port, (u32)index, &gid_entry);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to get gid, func_id(%d)\n", __func__, rdev->glb_func_id);
        return ret;
    }

    ret = memcpy_s((void *)gid->raw, sizeof(*gid), (void *)gid_entry.raw, sizeof(*gid));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memcpy gid.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    // 按照OFED的gid生成方式转换GID, 仅IPv4场景需要转换
    if (gid_entry.dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) {
        // 未add的gid直接返回，不需要转换
        if ((gid->global.subnet_prefix == 0) && (gid->global.interface_id == 0)) {
            return 0;
        }
        ipv6_addr_set_v4mapped(*((u32 *)(void *)gid + ROCE_GID_IP_IDX), (struct in6_addr *)gid);
    }

    return 0;
}

/* ****************************************************************************
 Prototype    : roce3_query_pkey
 Description  : query pkey
 Input        : struct ib_device *ibdev
                u8 port
                u16 index
                u16 *pkey
 Output       : u16 *pkey

  1.Date         : 2015/5/8
    Modification : Created function

**************************************************************************** */
#if defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
int roce3_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
#else
int roce3_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
#endif
{
    struct roce3_device *rdev = NULL;
    if ((ibdev == NULL) || (pkey == NULL)) {
        pr_err("[ROCE] %s: Ibdev or pkey is null\n", __func__);
        return -EINVAL;
    }

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }

    if (pkey == NULL) {
        pr_err("[ROCE] %s: Pkey is null\n", __func__);
        return -EINVAL;
    }

    *pkey = 0xffff;

    return 0;
}

/* ****************************************************************************
 Prototype    : roce3_modify_device
 Description  : modify device attribute
 Input        : struct ib_device *ibdev
                int mask
                struct ib_device_modify *props
 Output       : None

  1.Date         : 2015/5/8
    Modification : Created function

**************************************************************************** */
int roce3_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props)
{
    unsigned long flags = 0;
    struct roce3_device *rdev = NULL;
    int ret;

    if ((ibdev == NULL) || (props == NULL)) {
        pr_err("[ROCE] %s: Ibdev or props is null\n", __func__);
        return -EINVAL;
    }

    if (((unsigned int)mask & ~IB_DEVICE_MODIFY_NODE_DESC) != 0) {
        pr_err("[ROCE] %s: Not supported to modify node description\n", __func__);
        return -EOPNOTSUPP;
    }

    if ((((u32)mask) & IB_DEVICE_MODIFY_NODE_DESC) == 0) {
        pr_info("[ROCE] %s: No need to modify node description\n", __func__);
        return 0;
    }

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }

    spin_lock_irqsave(&rdev->node_desc_lock, flags);
    ret =
        memcpy_s((void *)ibdev->node_desc, IB_DEVICE_NODE_DESC_MAX, (void *)props->node_desc, IB_DEVICE_NODE_DESC_MAX);
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memcpy node_desc.(ret:%d)\n", __func__, ret);
        spin_unlock_irqrestore(&rdev->node_desc_lock, flags);
        return -ENOMEM;
    }
    spin_unlock_irqrestore(&rdev->node_desc_lock, flags);

    return 0;
}

/* ****************************************************************************
 Prototype    : roce3_modify_port
 Description  : modify port attribute
 Input        : struct ib_device *ibdev
                u8 port
                int mask
                struct ib_port_modify *props
 Output       : None

  1.Date         : 2015/5/8
    Modification : Created function

**************************************************************************** */
#if defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
int roce3_modify_port(struct ib_device *ibdev, u32 port, int mask, struct ib_port_modify *props)
#else
int roce3_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props)
#endif
{
    int ret = 0;
    struct ib_port_attr attr;
    struct roce3_device *rdev = NULL;

    if (ibdev == NULL) {
        pr_err("[ROCE] %s: Ibdev is null\n", __func__);
        return -EINVAL;
    }

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }

    ret = memset_s(&attr, sizeof(struct ib_port_attr), 0, sizeof(struct ib_port_attr));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memset attr.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    mutex_lock(&rdev->cap_mask_mutex);

    ret = roce3_query_port(ibdev, port, &attr);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to query port, func_id(%d)\n", __func__, rdev->glb_func_id);
    }

    mutex_unlock(&rdev->cap_mask_mutex);

    return ret;
}

static void roce3_alloc_ucontext_set(struct roce3_device *rdev, struct roce3_alloc_ucontext_resp *resp)
{
    struct rdma_service_cap *rdma_cap = NULL;

    rdma_cap = &rdev->rdma_cap;

    resp->num_qps = rdma_cap->dev_rdma_cap.roce_own_cap.max_qps;
    resp->num_xsrqs = rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs;
    resp->cqe_size = rdma_cap->cqe_size;
    resp->wqebb_size = rdma_cap->wqebb_size;
    resp->dwqe_size = rdma_cap->direct_wqe_size;
    resp->max_msg_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_msg_sz;
    resp->max_comp_vector = rdma_cap->num_comp_vectors;
    resp->max_inline_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz;

    resp->storage_aa_en = roce3_is_roceaa(rdev->cfg_info.scence_id);
    resp->phy_port = rdev->hw_info.phy_port;
    resp->srq_container_en = rdev->cfg_info.srq_container_en;
    resp->srq_container_mode = rdev->cfg_info.srq_container_mode;
    resp->xrc_srq_container_mode = rdev->cfg_info.xrc_srq_container_mode;
    resp->warn_th = rdev->cfg_info.warn_th;

    roce3_resp_set_ext(rdev, resp);
    return;
}

static int roce3_alloc_ucontext_pre_check(struct ib_device *ibdev, const struct ib_udata *udata)
{
    struct roce3_device *rdev = NULL;

    if ((ibdev == NULL) || (udata == NULL)) {
        pr_err("[ROCE] %s: Ibdev or udata is null\n", __func__);
        return -EINVAL;
    }

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }

    if (!rdev->ib_active) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Device is abnormal, func_id(%d)\n", __func__, rdev->glb_func_id);
        return -EAGAIN;
    }

    return 0;
}

static int roce3_alloc_ucontext_return(struct roce3_device *rdev, struct ib_udata *udata,
    struct roce3_ucontext *context, struct roce3_alloc_ucontext_resp *resp)
{
    int ret;

    resp->db_offset = context->db_dma_addr & ((1 << PAGE_SHIFT) - 1);
    resp->dwqe_offset = context->dwqe_dma_addr & ((1 << PAGE_SHIFT) - 1);

    if (context->dwqe_dma_addr == 0) {
        resp->dwqe_size = 0;
    }

    roce3_ucontext_set_ext(rdev, context);

    INIT_LIST_HEAD(&context->db_page_list);
    mutex_init(&context->db_page_mutex);

    /* Copy data to user space */
    ret = ib_copy_to_udata_ext(udata, resp);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to copy data to user space, func_id(%d)\n", __func__,
            rdev->glb_func_id);
        return ret;
    }

    return 0;
}
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)) || defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) \
    || defined(CENTOS8_2_4_18_0)
int roce3_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)
{
    int ret;
    struct roce3_ucontext *context = rdma_udata_to_drv_context(
        udata, struct roce3_ucontext, ibucontext);
    struct roce3_device *rdev = to_roce3_dev(ibucontext->device);
    struct roce3_alloc_ucontext_resp *resp = NULL;

    ret = roce3_alloc_ucontext_pre_check(ibucontext->device, udata);
    if (ret != 0) {
        return ret;
    }

    resp = roce3_resp_alloc_ext();
    if (resp == NULL) {
        ret = (-ENOMEM);
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc ucontext, func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        goto err;
    }

    roce3_alloc_ucontext_set(rdev, resp);
    /* Alloc user space context Doorbell and DWQE */
    ret = hinic3_alloc_db_phy_addr(rdev->hwdev, &context->db_dma_addr, &context->dwqe_dma_addr);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc DB pa, ret(%d), func_id(%hu)\n", __func__, ret,
            rdev->glb_func_id);
        goto err_db;
    }

    /* Copy data to user space */
    ret = roce3_alloc_ucontext_return(rdev, udata, context, resp);
    if (ret != 0) {
        goto err_return;
    }

    kfree(resp);

    return 0;
err_return:
    hinic3_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, context->dwqe_dma_addr);
err_db:
    kfree(resp);
err:
    return ret;
}

void roce3_dealloc_ucontext(struct ib_ucontext *ibcontext)
{
    struct roce3_ucontext *context = NULL;
    struct roce3_device *rdev = NULL;

    if (ibcontext == NULL) {
        pr_err("[ROCE] %s: Ibcontext is null\n", __func__);
        return;
    }

    context = to_roce3_ucontext(ibcontext);
    rdev = to_roce3_dev(ibcontext->device);

    hinic3_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, context->dwqe_dma_addr);

    return;
}
#else
struct ib_ucontext *roce3_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
{
    int ret = 0;
    struct roce3_device *rdev = NULL;
    struct roce3_ucontext *context = NULL;
    struct roce3_alloc_ucontext_resp *resp = NULL;

    ret = roce3_alloc_ucontext_pre_check(ibdev, udata);
    if (ret != 0) {
        return (struct ib_ucontext *)ERR_PTR((long)ret);
    }

    rdev = to_roce3_dev(ibdev);

    resp = roce3_resp_alloc_ext();
    if (resp == NULL) {
        ret = (-ENOMEM);
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc ucontext, func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        goto err_resp_alloc;
    }

    roce3_alloc_ucontext_set(rdev, resp);

    context = roce3_ucontext_alloc_ext();
    if (context == NULL) {
        ret = (-ENOMEM);
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc ucontext, func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        goto err_ucontext_alloc;
    }

    /* Alloc user space context Doorbell and DWQE */
    ret = hinic3_alloc_db_phy_addr(rdev->hwdev, &context->db_dma_addr, &context->dwqe_dma_addr);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc DB pa, ret(%d), func_id(%hu)\n", __func__, ret,
            rdev->glb_func_id);
        goto err_db_phy_addr_alloc;
    }

    /* Copy data to user space */
    ret = roce3_alloc_ucontext_return(rdev, udata, context, resp);
    if (ret != 0) {
        goto err_return;
    }

    kfree(resp);

    return (struct ib_ucontext *)&context->ibucontext;

err_return:
    hinic3_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, context->dwqe_dma_addr);
err_db_phy_addr_alloc:
    kfree(context);
err_ucontext_alloc:
    kfree(resp);
err_resp_alloc:
    return (struct ib_ucontext *)ERR_PTR((long)ret);
}

/* ****************************************************************************
 Prototype    : roce3_dealloc_ucontext
 Description  : dealloc ucontext
 Input        : struct ib_ucontext *ibcontext
 Output       : None

  1.Date         : 2015/5/8
    Modification : Created function

**************************************************************************** */
int roce3_dealloc_ucontext(struct ib_ucontext *ibcontext)
{
    struct roce3_ucontext *context = NULL;
    struct roce3_device *rdev = NULL;

    if (ibcontext == NULL) {
        pr_err("[ROCE] %s: Ibcontext is null\n", __func__);
        return -EINVAL;
    }

    context = to_roce3_ucontext(ibcontext);
    rdev = to_roce3_dev(ibcontext->device);

    hinic3_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, context->dwqe_dma_addr);
    kfree(context);

    return 0;
}
#endif

/* ****************************************************************************
 Prototype    : roce3_mmap
 Description  : memory map
 Input        : struct ib_ucontext *ibcontext
                struct vm_area_struct *vma
 Output       : None

  1.Date         : 2015/5/8
    Modification : Created function

**************************************************************************** */
int roce3_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
{
    struct roce3_device *rdev = NULL;
    struct roce3_ucontext *ucontext = NULL;
    unsigned long db_pfn = 0;
    unsigned long dwqe_pfn = 0;
    int res = 0;

    if ((ibcontext == NULL) || (vma == NULL)) {
        pr_err("[ROCE] %s: Ibcontext or vma is null\n", __func__);
        return -EINVAL;
    }

    rdev = to_roce3_dev(ibcontext->device);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }

    ucontext = to_roce3_ucontext(ibcontext);
    db_pfn = ucontext->db_dma_addr >> PAGE_SHIFT;
    dwqe_pfn = ucontext->dwqe_dma_addr >> PAGE_SHIFT;

    if ((vma->vm_end - vma->vm_start) != PAGE_SIZE) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: (Vm_end - vm_start) is not equal to PAGE_SIZE, func_id(%d)\n",
            __func__, rdev->glb_func_id);
        return -EINVAL;
    }

    /* map hw DB to physical page from user */
    if (vma->vm_pgoff == USR_MMAP_DB_OFFSET) {
        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
        /* construct vm_start~vm_start+PAGE_SIZE page table
         * db_pfn is page number
         * vm_page_prot means attr
         */
        if (io_remap_pfn_range(vma, vma->vm_start, db_pfn, PAGE_SIZE, vma->vm_page_prot) != 0) {
            dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to do db io remap, func_id(%d)\n", __func__,
                rdev->glb_func_id);
            return -EAGAIN;
        }
        return 0;
    }

    if ((vma->vm_pgoff == USR_MMAP_DWQE_OFFSET) && (rdev->rdma_cap.direct_wqe_size != 0)) { // DWQE mmap
#ifdef __aarch64__
        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
#else
        vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
#endif

        if (io_remap_pfn_range(vma, vma->vm_start, dwqe_pfn, PAGE_SIZE, vma->vm_page_prot) != 0) {
            dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to do dwqe io remap, func_id(%d)\n", __func__,
                rdev->glb_func_id);
            return -EAGAIN;
        }
        return 0;
    }

    res = roce3_mmap_ext(rdev, ucontext, vma);
    return res;
}
#if defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
enum rdma_link_layer roce3_port_link_layer(struct ib_device *ibdev, u32 port_num)
#else
enum rdma_link_layer roce3_port_link_layer(struct ib_device *ibdev, u8 port_num)
#endif
{
    struct roce3_device *rdev = NULL;

    if (ibdev == NULL) {
        return IB_LINK_LAYER_UNSPECIFIED;
    }

    rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return IB_LINK_LAYER_UNSPECIFIED;
    }

    if (port_num != ROCE_DEFAULT_PORT_NUM) {
        return IB_LINK_LAYER_UNSPECIFIED;
    }

    return IB_LINK_LAYER_ETHERNET;
}

static void roce3_resolve_cb(int status, struct sockaddr *src_addr, struct rdma_dev_addr *addr, void *context)
{
    ((struct roce3_resolve_cb_context *)context)->status = status;
    complete(&((struct roce3_resolve_cb_context *)context)->comp);
}

static int roce3_rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid, const union ib_gid *dgid, u8 *dmac,
    const struct net_device *ndev, int *hoplimit, struct roce3_device *rdev)
{
    struct rdma_dev_addr dev_addr;
    struct roce3_resolve_cb_context ctx;
    union {
        struct sockaddr _sockaddr;
        struct sockaddr_in _sockaddr_in;
        struct sockaddr_in6 _sockaddr_in6;
    } sgid_addr, dgid_addr;
    int ret;

    rdma_gid2ip(&sgid_addr._sockaddr, sgid);
    rdma_gid2ip(&dgid_addr._sockaddr, dgid);

    ret = memset_s(&dev_addr, sizeof(dev_addr), 0, sizeof(dev_addr));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memset dev_addr.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    if (ndev) {
        dev_addr.bound_dev_if = ndev->ifindex;
        dev_addr.net = dev_net(ndev);
    } else {
        dev_addr.net = current->nsproxy->net_ns;
    }

    init_completion(&ctx.comp);

#if (LINUX_VERSION_CODE == KERNEL_VERSION(4, 18, 0)) || LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) \
    || defined(OFED_MLNX_5_7_8) || defined(CENTOS8_2_4_18_0)
    ret = rdma_resolve_ip(&sgid_addr._sockaddr, &dgid_addr._sockaddr, &dev_addr, RESOLVE_IP_TIME_OUT,
        roce3_resolve_cb, false, &ctx);
#elif LINUX_VERSION_CODE == KERNEL_VERSION(4, 19, 90)
    ret = rdma_resolve_ip(&sgid_addr._sockaddr, &dgid_addr._sockaddr, &dev_addr, RESOLVE_IP_TIME_OUT,
        roce3_resolve_cb, &ctx);
#else
    ret = rdma_resolve_ip(&roce3_self, &sgid_addr._sockaddr, &dgid_addr._sockaddr, &dev_addr, RESOLVE_IP_TIME_OUT,
        roce3_resolve_cb, &ctx);
#endif
    if (ret != 0) {
        pr_err("[ROCE] %s: rdma_resolve_ip failed. Igonore the err.\n", __func__);
        roce3_resolve_cb(0, &sgid_addr._sockaddr, &dev_addr, &ctx);
    }

    wait_for_completion(&ctx.comp);

    ret = memcpy_s(dmac, ETH_ALEN, dev_addr.dst_dev_addr, ETH_ALEN);
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memcpy dmac.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }
    if (hoplimit) {
        *hoplimit = dev_addr.hoplimit;
    }

    return 0;
}

static int roce3_ah_valid_check(struct ib_global_route *grh, u16 *vlan_id, u8 *dmac)
{
    u8 unicast_gid0[ROCE_GID_LEN] = { 0 };
    u8 unicast_gid1[ROCE_GID_LEN] = { 0 };

    /* check gid(unicast gid can not be 0 or 1) */
    unicast_gid0[ROCE_GID_HIGHEST_BYTE] = 0;
    unicast_gid1[ROCE_GID_HIGHEST_BYTE] = 1;

    if ((ROCE_MEMCMP(grh->dgid.raw, unicast_gid0, sizeof(union ib_gid)) == 0) ||
        (ROCE_MEMCMP(grh->dgid.raw, unicast_gid1, sizeof(union ib_gid)) == 0)) {
        pr_err("[ROCE] %s: Invalid unicast dgid\n", __func__);
        return (-EINVAL);
    }

    if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) != 0) {
        rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw, dmac);
        *vlan_id = ROCE_DEFAULT_VLAN_ID;
    }

    return 0;
}

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)) || (LINUX_VERSION_CODE == KERNEL_VERSION(4, 19, 90)) \
    || (LINUX_VERSION_CODE == KERNEL_VERSION(4, 18, 0)) || defined(OFED_MLNX_5_7_8)
static int roce3_fill_gid_attr(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, union ib_gid *sgid,
    const struct ib_gid_attr **sgid_attr)
{
    int ret = 0;

    ret = rdma_query_gid(&rdev->ib_dev, ah_attr->port_num, ah_attr->grh.sgid_index, sgid);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to query gid func_id(%hu),port_num(%d),gid_index(%d),ret(%d)\n",
            rdev->glb_func_id, ah_attr->port_num, ah_attr->grh.sgid_index, ret);
        return ret;
    }

    *sgid_attr = rdma_get_gid_attr(&rdev->ib_dev, ah_attr->port_num, ah_attr->grh.sgid_index);
    if (IS_ERR_OR_NULL(*sgid_attr)) {
        ret = (int)PTR_ERR(*sgid_attr);
        dev_err(rdev->hwdev_hdl, "[ROCE] : Failed to get sgid_attr, func_id(%hu), ret(%d).\n", rdev->glb_func_id, ret);
        return ret;
    }

    return ret;
}

static void roce3_release_gid_ref_cnt(const struct ib_gid_attr *sgid_attr)
{
    rdma_put_gid_attr(sgid_attr);
}
#else
static int roce3_fill_gid_attr(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, union ib_gid *sgid,
    const struct ib_gid_attr **sgid_attr)
{
    int ret = 0;
    struct ib_gid_attr *temp_sgid_attr = NULL;

    temp_sgid_attr = (struct ib_gid_attr *)kzalloc(sizeof(struct ib_gid_attr), GFP_KERNEL);
    if (temp_sgid_attr == NULL) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to kzalloc in roce3_fill_gid_attr\n");
        return -ENOMEM;
    }

    ret = ib_query_gid(&rdev->ib_dev, ah_attr->port_num, ah_attr->grh.sgid_index, sgid, temp_sgid_attr);
    if ((ret != 0) || (temp_sgid_attr->ndev == 0)) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to query gid func_id(%hu),port_num(%d),gid_index(%d),ret(%d)\n",
            rdev->glb_func_id, ah_attr->port_num, ah_attr->grh.sgid_index, ret);

        if (ret == 0) {
            ret = -ENXIO;
        }
        return ret;
    }
    *sgid_attr = temp_sgid_attr;

    return ret;
}

static void roce3_release_gid_ref_cnt(const struct ib_gid_attr *sgid_attr)
{
    dev_put(sgid_attr->ndev);
}
#endif

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)) || defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4)
static struct net_device *roce3_fill_netdev(struct roce3_device *rdev, union ib_gid *sgid)
{
    struct net_device *netdev = NULL;
    union {
        struct sockaddr _sockaddr;
        struct sockaddr_in _sockaddr_in;
        struct sockaddr_in6 _sockaddr_in6;
    } socket_addr;

    rdma_gid2ip(&socket_addr._sockaddr, sgid);

    /* find netdev,rdev->ndevis not valid in vlan scenario */
    netdev = ip_dev_find(&init_net, ((const struct sockaddr_in *)&socket_addr._sockaddr)->sin_addr.s_addr);
    if (netdev) {
        dev_put(netdev);
    }

    return netdev;
}
#else
static struct net_device *roce3_fill_netdev(struct roce3_device *rdev, union ib_gid *sgid)
{
    return NULL;
}
#endif

int roce3_resolve_grh(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, u16 *vlan_id, struct ib_udata *udata)
{
    int ret = 0;
    u8 zero_mac[ETH_ALEN] = { 0 };
    u8 *dmac = NULL;
    union ib_gid sgid;
    const struct ib_gid_attr *sgid_attr = NULL;
    struct net_device *netdev = NULL;

    if ((rdev == NULL) || (ah_attr == NULL) || (vlan_id == NULL)) {
        pr_err("[ROCE, ERR] %s: Input pointer is NULL, rdev(%px), ah_attr(%px), vlan_id(%px).\n", __func__,
            rdev, ah_attr, vlan_id);
        return (-EINVAL);
    }

    dmac = ah_attr->roce.dmac;
    ret = roce3_ah_valid_check(&ah_attr->grh, vlan_id, dmac);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to check grh input, func_id(%hu), ret(%d).\n",
            rdev->glb_func_id, ret);
        return ret;
    }
    if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) != 0) {
        return 0;
    }

    ret = roce3_fill_gid_attr(rdev, ah_attr, &sgid, &sgid_attr);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to fill gid attr, func_id(%hu), ret(%d)\n", rdev->glb_func_id,
            ret);
        return ret;
    }

    netdev = roce3_fill_netdev(rdev, &sgid);

    /* reparse dmac avoiding invalid damc from OFED */
    ret = roce3_rdma_addr_find_l2_eth_by_grh(&sgid, &ah_attr->grh.dgid, dmac, netdev, NULL, rdev);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to find dmac by grh, func_id(%hu)\n", rdev->glb_func_id);
        goto resolve_grh_end;
    }

    if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Failed to find valid dmac, func_id(%hu)\n", rdev->glb_func_id);
        ret = (-EINVAL);
        goto resolve_grh_end;
    }

    *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev);

resolve_grh_end:
    roce3_release_gid_ref_cnt(sgid_attr);

    return ret;
}

static int ah_get_vlan_id(struct roce3_device *rdev, struct ib_pd *pd, struct rdma_ah_attr *ah_attr, u32 *vlan_id)
{
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)) || (LINUX_VERSION_CODE == KERNEL_VERSION(4, 19, 90)) \
    || (LINUX_VERSION_CODE == KERNEL_VERSION(4, 18, 0)) || defined(OFED_MLNX_5_7_8) || defined(CENTOS8_2_4_18_0)
	struct net_device *ndev;

    rcu_read_lock();
    ndev = rcu_dereference(ah_attr->grh.sgid_attr->ndev);
    if (ndev == NULL) {
        rcu_read_unlock();
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] : Net device is NULL, func_id(%hu)\n", rdev->glb_func_id);
        return -EINVAL;
    }
    *vlan_id = rdma_vlan_dev_vlan_id(ndev);
    rcu_read_unlock();
#else
    struct ib_gid_attr gid_attr;
    union ib_gid sgid;
    int ret;
    /* Get SGID */
    ret = ib_get_cached_gid(pd->device, rdma_ah_get_port_num(ah_attr), ah_attr->grh.sgid_index, &sgid, &gid_attr);
    if (ret != 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: Failed to get sgid, ret(%d), func_id(%d)\n", __func__, ret,
            rdev->glb_func_id);
        return -EFAULT;
    }

    *vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
    dev_put(gid_attr.ndev);
#endif
    return 0;
}

static int create_ib_ah_kernel(struct roce3_device *rdev, struct ib_pd *pd, struct roce3_ah *rah,
    struct rdma_ah_attr *ah_attr)
{
    int ret;
    u8 *dmac = ah_attr->roce.dmac;
    u32 vlan_id = 0;

    ret = ah_get_vlan_id(rdev, pd, ah_attr, &vlan_id);
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to get vlan_id (ret:%d)\n", __func__, ret);
        return -EFAULT;
    }

    if (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) != 0) {
        ret = memcpy_s((void *)rah->priv_ah.dgid, sizeof(rah->priv_ah.dgid), (void *)ah_attr->grh.dgid.raw,
            sizeof(rah->priv_ah.dgid));
        if (ret != 0) {
            pr_err("[ROCE, ERR] %s: Failed to memcpy dgid.(ret:%d)\n", __func__, ret);
            return -ENOMEM;
        }
        rah->priv_ah.dw2.bs.flow_label = ah_attr->grh.flow_label & 0xfffff;
        rah->priv_ah.dw1.bs.sgid_index = ah_attr->grh.sgid_index & 0x7f;
        rah->priv_ah.dw1.bs.hoplimit = ah_attr->grh.hop_limit;
        rah->priv_ah.dw1.bs.tclass = (u8)(ah_attr->grh.traffic_class | 0x2);
    }
    rah->priv_ah.dw0.bs.pd = to_roce3_pd(pd)->pdn & 0x3ffff;
    rah->priv_ah.dw0.bs.wqe_cos = roce3_get_db_cos_from_vlan_pri(rdev, ah_attr->sl);
    rah->priv_ah.dw0.value = cpu_to_be32(rah->priv_ah.dw0.value);

    rah->priv_ah.dw1.bs.port = ah_attr->port_num & 0xf;
    rah->priv_ah.dw2.bs.smac_index = rdev->glb_func_id; /* set global Function ID */
    rah->priv_ah.dw2.value = cpu_to_be32(rah->priv_ah.dw2.value);

    rah->priv_ah.dw1.bs.resv = 0;
    rah->priv_ah.dw7.bs.vlan_id = vlan_id & 0xfff;
    rah->priv_ah.dw7.bs.vlan_pri = ah_attr->sl & 0x7;

    rah->priv_ah.dw1.value = cpu_to_be32(rah->priv_ah.dw1.value);

    rah->priv_ah.dw7.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | dmac[1];
    rah->priv_ah.dw7.value = cpu_to_be32(rah->priv_ah.dw7.value);

    ret = memcpy_s((void *)&rah->priv_ah.dmac_l32, sizeof(rah->priv_ah.dmac_l32),
        (void *)&dmac[ROCE_RAH_DMAC_L32_START], sizeof(rah->priv_ah.dmac_l32));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memcpy dmac_l32.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    return 0;
}

static int create_ib_ah_user(struct roce3_device *rdev, struct rdma_ah_attr *ah_attr, struct ib_udata *udata)
{
    struct roce3_create_ah_ib_resp resp = {};
    roce3_get_cos_inbuf_s dcb_inbuf = {};
    int ret;

    ret = memset_s(ah_attr->roce.dmac, sizeof(ah_attr->roce.dmac), 0, sizeof(ah_attr->roce.dmac));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memset dmac.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    ret = roce3_resolve_grh(rdev, ah_attr, &resp.vlan_id, udata);
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to resolve grh.(ret:%d)\n", __func__, ret);
        return -EINVAL;
    }

    ret = memcpy_s(resp.dmac, ETH_ALEN, ah_attr->roce.dmac, ETH_ALEN);
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memcpy_s ah dmac.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    dcb_inbuf.sl = ah_attr->sl;
    dcb_inbuf.sgid_index = ah_attr->grh.sgid_index;
    dcb_inbuf.port_num = ah_attr->port_num;
    dcb_inbuf.traffic_class = ah_attr->grh.traffic_class;
    ret = roce3_get_dcb_cfg_cos(rdev, &dcb_inbuf, &resp.cos);
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to get cos from dcb info.(ret:%d)\n", __func__, ret);
        return ret;
    }

    ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to copy data to user.(ret:%d)\n", __func__, ret);
    }

    return ret;
}

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)) || defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) \
    || defined(FEATURE_CREATE_AH_4)
#if defined(FEATURE_CREATE_AH_4)
int roce3_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr, u32 flags, struct ib_udata *udata)
#else
int roce3_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata)
#endif
{
    struct roce3_ah *rah = to_roce3_ah(ibah);
    struct roce3_device *rdev = to_roce3_dev(ibah->device);
#if !defined(FEATURE_NO_NETDEV_EIN_PAGE)
    struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
#endif
    enum rdma_ah_attr_type ah_type = ah_attr->type;
    int ret;

    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }

    if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) && (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) == 0)) {
        return -EINVAL;
    }

    if (udata != NULL) {
        ret = create_ib_ah_user(rdev, ah_attr, udata);
    } else {
        ret = create_ib_ah_kernel(rdev, ibah->pd, rah, ah_attr);
    }

    return ret;
}
#else
#if LINUX_VERSION_CODE == KERNEL_VERSION(4, 18, 0)
struct ib_ah *roce3_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr, u32 flags, struct ib_udata *udata)
#else
struct ib_ah *roce3_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr, struct ib_udata *udata)
#endif
{
    struct roce3_ah *rah = NULL;
    struct roce3_device *rdev = to_roce3_dev(pd->device);
    enum rdma_ah_attr_type ah_type = ah_attr->type;
    int ret;

    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return ERR_PTR(-EPERM);
    }

    if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) && (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) == 0)) {
        return ERR_PTR(-EINVAL);
    }

    rah = kzalloc(sizeof(*rah), GFP_ATOMIC);
    if (!rah) {
        dev_err(rdev->hwdev_hdl, "[ROCE, ERR] %s: Failed to alloc ah, func_id(%d)\n", __func__, rdev->glb_func_id);
        return ERR_PTR(-ENOMEM);
    }

    if (udata != NULL) {
        ret = create_ib_ah_user(rdev, ah_attr, udata);
    } else {
        ret = create_ib_ah_kernel(rdev, pd, rah, ah_attr);
    }
    if (ret) {
        kfree(rah);
        return ERR_PTR(ret);
    }

    return &rah->ibah;
}
#endif

int roce3_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
{
    struct roce3_ah *ah = NULL;
    struct roce3_priv_ah priv_ah;
    int ret;

    if ((ibah == NULL) || (ah_attr == NULL)) {
        pr_err("[ROCE] %s: Ibah or ah_attr is null\n", __func__);
        return -EINVAL;
    }

    ah = to_roce3_ah(ibah);
    ret = memset_s(ah_attr, sizeof *ah_attr, 0, sizeof *ah_attr);
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memset ah_attr.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }

    priv_ah.dw1.value = be32_to_cpu(ah->priv_ah.dw1.value);
    priv_ah.dw2.value = be32_to_cpu(ah->priv_ah.dw2.value);
    priv_ah.dw7.value = be32_to_cpu(ah->priv_ah.dw7.value);

    ah_attr->ah_flags = IB_AH_GRH;
    ah_attr->sl = priv_ah.dw7.bs.vlan_pri;
    ah_attr->port_num = priv_ah.dw1.bs.port;
    ah_attr->grh.traffic_class = priv_ah.dw1.bs.tclass;
    ah_attr->grh.hop_limit = priv_ah.dw1.bs.hoplimit;
    ah_attr->grh.sgid_index = priv_ah.dw1.bs.sgid_index;
    ah_attr->grh.flow_label = priv_ah.dw2.bs.flow_label;

    ret = memcpy_s((void *)ah_attr->grh.dgid.raw, sizeof(ah->priv_ah.dgid), (void *)ah->priv_ah.dgid,
        sizeof(ah->priv_ah.dgid));
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to memcpy dgid.(ret:%d)\n", __func__, ret);
        return -ENOMEM;
    }
    return 0;
}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) || defined(OFED_MLNX_5_7_8)
int roce3_destroy_ah(struct ib_ah *ibah, u32 flags)
{
    return 0;
}
#elif defined(OFED_MLNX_5_4) || defined(FEATURE_DESTROY_CQ_VOI)
void roce3_destroy_ah(struct ib_ah *ibah, u32 flags) {}
#else
#if LINUX_VERSION_CODE == KERNEL_VERSION(4, 18, 0)
int roce3_destroy_ah(struct ib_ah *ibah, u32 flags)
#else
int roce3_destroy_ah(struct ib_ah *ibah)
#endif
{
    struct roce3_ah *rah = NULL;

    if (ibah == NULL) {
        pr_err("[ROCE] %s: Ibah is null\n", __func__);
        return -EINVAL;
    }

    rah = to_roce3_ah(ibah);

    kfree(rah);

    return 0;
}
#endif
#if defined(OFED_MLNX_5_7_8) || defined(OFED_MLNX_5_4) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
int roce3_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable)
#else
int roce3_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable)
#endif
{
    struct ib_port_attr attr;
    int err;
    struct roce3_device *rdev = to_roce3_dev(ibdev);
    if (roce3_hca_is_present(rdev) == 0) {
        dev_err(rdev->hwdev_hdl, "[ROCE] %s: HCA not present(return fail), func_id(%hu)\n", __func__,
            rdev->glb_func_id);
        return -EPERM;
    }
    immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; // only rocev2

    err = ib_query_port(ibdev, port_num, &attr);
    if (err != 0) {
        pr_err("[ROCE] %s: query ib port failed\n", __func__);
        return err;
    }

    immutable->pkey_tbl_len = attr.pkey_tbl_len;
    immutable->gid_tbl_len = attr.gid_tbl_len;
    immutable->max_mad_size = IB_MGMT_MAD_SIZE;

    return 0;
}

int roce3_get_dcb_cfg_cos(struct roce3_device *rdev, roce3_get_cos_inbuf_s *inbuf, u8 *cos)
{
    int ret;
    u8 pri;
    struct rdma_gid_entry gid;
    struct hinic3_dcb_state dcb = {};
    u8 tclass = inbuf->traffic_class;
    struct rdma_ah_attr ah = {};

    ret = roce3_get_gid(rdev, inbuf->port_num, inbuf->sgid_index, &gid);
    if (ret != 0) {
        pr_err("[ROCE, ERR] %s: Failed to init gid info\n", __func__);
        return (-EINVAL);
    }

    if (hinic3_func_type(rdev->hwdev) == TYPE_VF) {
        ret = hinic3_get_pf_dcb_state(rdev->hwdev, &dcb);
    } else {
        ret = hinic3_get_dcb_state(rdev->hwdev, &dcb);
    }
    
    if (ret != 0) {
        pr_err("[ROCE] %s: hinic3_get_dcb_state failed.ret: %d.\n", __func__, ret);
        return (-EINVAL);
    }

    *cos = dcb.default_cos;
    gid.dw6_h.value = cpu_to_le16(gid.dw6_h.value);
    if ((dcb.trust == ROCE3_DCB_PCP) && (gid.dw6_h.bs.tag == ROCE_GID_VLAN_INVALID)) {
        // pcp cfg & no vlan should use default cos
        return 0;
    }

    if (inbuf->port_num >= 1 && inbuf->port_num <= rdev->hw_info.config_num_ports) {
        ah.type = RDMA_AH_ATTR_TYPE_ROCE;
        ah.grh.sgid_index = inbuf->sgid_index;
        ah.grh.sgid_attr = rdma_get_gid_attr(&rdev->ib_dev, inbuf->port_num, inbuf->sgid_index);
        if (ah.grh.sgid_attr != NULL) {
            (void)memcpy_s(ah.grh.dgid.raw, sizeof(ah.grh.dgid.raw), inbuf->dgid_raw, sizeof(inbuf->dgid_raw));
            tclass_get_tclass(rdev, &rdev->tcd[inbuf->port_num - 1], &ah, inbuf->port_num, &tclass);
        }
    }

    pri = (dcb.trust == ROCE3_DCB_PCP) ? inbuf->sl : (tclass >> ROCE3_DSCP_IDX);
    ret = hinic3_get_cos_by_pri(rdev->hwdev, pri, cos);
    if (ret != 0) {
        pr_err("[ROCE] %s: get_cos_by_pri failed.ret: %d, pri:%u, dcb_on:%u, trust:%u.\n", __func__, ret, pri,
            dcb.dcb_on, dcb.trust);
        return (-EINVAL);
    }

    return 0;
}
